package main_app;

import java.io.IOException;
import java.util.Map;

import output_handler.OutputMngr;
import parser.Parser;
import smoothing.HeldOut;
import smoothing.Lidstone;

public class MainApp {

	private static final String LIDSTONE_TRAIN_FILE_NAME = "lidstone_train_set.txt";
	private static final String LIDSTONE_VALID_FILE_NAME = "lidstone_validation_set.txt";
	private static final String HELDOUT_TRAIN_FILE_NAME = "heldout_train_set.txt";
	private static final String HELDOUT_FILE_NAME = "heldout_set.txt";
	private static final double LIDSTONE_SPLIT_PROP = 0.9;
	private static final double REPORT_SMOOTHING_PARAMETER = 0.1; // lambda value for the report
	private static final double HELDOUT_SPLIT_PROP = 0.5;
	private static final String HEADER_TEST_START = "<TEST";
	private static final String HEADER_TRAIN_START = "<TRAIN";

	private static double bestLambda;
	private static Map<String, Integer> obsVoc;
	
	/**
	 * Runs the experiment on given data sets and input word.
	 * Reports results in the output file
	 * @param develFileName name of development data set file
	 * @param testFileName name of test data set file
	 * @param inputWord word to estimate its probability
	 * @param outputFileName name of output file
	 * @throws IOException if had errors reading or writing files
	 */
	public static void run(String develFileName, String testFileName,
			String inputWord, String outputFileName) throws IOException {

		// initialize object to manage output
		OutputMngr outputMngr = new OutputMngr(outputFileName);
		
		// initialize output file content
		initOutput(develFileName, testFileName, inputWord, outputFileName, outputMngr);
		
		// process development set according to Lidstone method
		procLidstone(develFileName, testFileName, inputWord, outputMngr);
		
		// process development set according to Held-out method
		procHeldOut(develFileName, testFileName, inputWord, outputMngr);
		
		// close output file
		outputMngr.closeOutput();
	}

	/**
	 * Writes to output the initial data 
	 * @param develFileName name of development data set file
	 * @param testFileName name of test data set file
	 * @param inputWord word to estimate its probability
	 * @param outputFileName name of output file
	 * @param outputMngr used to write to output
	 */
	private static void initOutput(String develFileName, String testFileName,
			String inputWord, String outputFileName, OutputMngr outputMngr) {
		
		// write given parameters to output file
		outputMngr.addLine(develFileName);
		outputMngr.addLine(testFileName);
		outputMngr.addLine(inputWord);
		outputMngr.addLine(outputFileName);
		Parser.HEADER_START = HEADER_TRAIN_START;
		outputMngr.addLine(Parser.VOCAB_SIZE+"");
		outputMngr.addLine(((double)(1.0/Parser.VOCAB_SIZE))+"");
	}
	
	/**
	 * Processes the development set according to Lidstone smoothing method.
	 * Writes the results to output
	 * @param develFileName name of development data set file
	 * @param testFileName name of test data set file
	 * @param inputWord word to estimate its probability
	 * @param outputMngr used to write to output
	 * @throws IOException if had errors reading or writing files
	 */
	private static void procLidstone(String develFileName, String testFileName,
			String inputWord, OutputMngr outputMngr) throws IOException {

		// write number of events in development set
		outputMngr.addLine(Parser.numEvents(develFileName) + "");

		// split development set to train and validation sets
		Parser.splitDataSet(develFileName, LIDSTONE_TRAIN_FILE_NAME,
				LIDSTONE_VALID_FILE_NAME, LIDSTONE_SPLIT_PROP);

		// write number of events in validation set
		outputMngr.addLine(Parser.numEvents(LIDSTONE_VALID_FILE_NAME) + "");

		// write number of events in training set
		outputMngr.addLine(Parser.numEvents(LIDSTONE_TRAIN_FILE_NAME) + "");

		// generate observed vocabulary out of training set
		obsVoc = Parser.genObsVoc(LIDSTONE_TRAIN_FILE_NAME);

		// write number of different events in training set
		outputMngr.addLine(obsVoc.size() + "");

		// write number of instances of input word
		outputMngr.addLine(obsVoc.get(inputWord) + "");

		// initialize Lidstone smoothing object
		Lidstone lidstone = new Lidstone(LIDSTONE_TRAIN_FILE_NAME, obsVoc,
				REPORT_SMOOTHING_PARAMETER);
		
		System.out.println("debug for lidstone: "+lidstone.debug());// TODO -- delete

		// write MLE for input word without smoothing
		outputMngr.addLine(lidstone.mleTrain(inputWord) + "");

		// write MLE for an unseen word without smoothing
		outputMngr.addLine(lidstone.mleTrain() + "");

		// write MLE for given word with pre-defined smoothing parameter
		outputMngr.addLine(lidstone.mleModel(inputWord) + "");

		// write MLE for an unseen word with pre-defined smoothing parameter
		outputMngr.addLine(lidstone.mleModel() + "");
		
		
		// write perplexity on the validation set using lambda = 0.01
		outputMngr.addLine((Parser.perplexity(new Lidstone(LIDSTONE_TRAIN_FILE_NAME, obsVoc, 0.01),
				LIDSTONE_VALID_FILE_NAME) + ""));
		
		// write perplexity on the validation set using lambda = 0.1
		outputMngr.addLine((Parser.perplexity(new Lidstone(LIDSTONE_TRAIN_FILE_NAME, obsVoc, 0.1),
				LIDSTONE_VALID_FILE_NAME) + ""));
		
		// write perplexity on the validation set using lambda = 1.0
		outputMngr.addLine((Parser.perplexity(new Lidstone(LIDSTONE_TRAIN_FILE_NAME, obsVoc, 1.0),
				LIDSTONE_VALID_FILE_NAME) + ""));
		
		// get the value of lambda that minimize the perplexity on the validation set
		bestLambda = bestLambdaForPerplexity(0.0, 2.0);
		
		// write the value of lambda that minimize the perplexity on the validation set
		outputMngr.addLine( bestLambda + "");
		
		// write the minimized perplexity on the validation set using the best value we found
		outputMngr.addLine((Parser.perplexity(new Lidstone(LIDSTONE_TRAIN_FILE_NAME, obsVoc, bestLambda),
				LIDSTONE_VALID_FILE_NAME) + ""));
	}
	
	/**
	 * Processes the development set according to Held-out smoothing method.
	 * Writes the results to output 
	 * @param develFileName name of development data set file
	 * @param testFileName name of test data set file
	 * @param inputWord word to estimate its probability
	 * @param outputMngr used to write to output
	 * @throws IOException if had errors reading or writing files
	 */
	private static void procHeldOut(String develFileName, String testFileName, String inputWord,
			OutputMngr outputMngr) throws IOException {
		
		// split development set to train and held-out sets
		Parser.splitDataSet(develFileName, HELDOUT_TRAIN_FILE_NAME, HELDOUT_FILE_NAME, HELDOUT_SPLIT_PROP);
		
		// write number of events in train set
		outputMngr.addLine(Parser.numEvents(HELDOUT_TRAIN_FILE_NAME)+"");
		
		// write number of events in held-out set
		outputMngr.addLine(Parser.numEvents(HELDOUT_FILE_NAME)+"");
		
		// initialize Held-Out smoothing object
		HeldOut heldOut = new HeldOut(HELDOUT_TRAIN_FILE_NAME, HELDOUT_FILE_NAME);
		
		System.out.println("debug for held-out: "+heldOut.debug());// TODO -- delete
				
		// write estimated probability for input word
		outputMngr.addLine(heldOut.mleModel(inputWord) + "");		
		
		// write estimated probability for an unseen word
		outputMngr.addLine(heldOut.mleModel() + "");	
		
		// update the parser to the test set
		Parser.HEADER_START = HEADER_TEST_START;
		
		// write total number of events in the test set
		outputMngr.addLine(Parser.genObsVoc(HELDOUT_TRAIN_FILE_NAME).size() + "");
		
		// calculate and write the perplexity of the test set according to the Lidstone model with the best lambda
		double lidstonePerplexity = Parser.perplexity(new Lidstone(LIDSTONE_TRAIN_FILE_NAME, obsVoc, bestLambda),
				testFileName);
		outputMngr.addLine(lidstonePerplexity + "");
		
		// calculate and write the perplexity of the test set according to the Held-Out model
		double heldOutPerplexity = Parser.perplexity(heldOut, testFileName); // TODO-- came out Infinity?!?
		outputMngr.addLine(heldOutPerplexity + "");
		
		// write 'L' if the Lidstone model is a better language model then Held-Out on the test set,
		// otherwise write 'H'
		if (lidstonePerplexity < heldOutPerplexity)
			outputMngr.addLine("L");
		else
			outputMngr.addLine("H");
		
		// write the result of the methods
		writeTableResults(new Lidstone(LIDSTONE_TRAIN_FILE_NAME, obsVoc, bestLambda), heldOut, outputMngr);
		
	}
	
	/**
	 * Calculate the best lambda value according to the perplexity
	 * @param lower bound for the lambda range
	 * @param upper limit for the lambda range
	 * @return best lambda value
	 * @throws IOException if had errors reading files
	 */
	private static double bestLambdaForPerplexity(double lowerBound, double upperLimit) throws IOException
	{
		
		// generate observed vocabulary out of training set
		//Map<String, Integer> obsVoc = Parser.genObsVoc(LIDSTONE_TRAIN_FILE_NAME);
		
		double minPerplexity = Parser.perplexity(new Lidstone(LIDSTONE_TRAIN_FILE_NAME,
				obsVoc, lowerBound), LIDSTONE_VALID_FILE_NAME);
		double bestLambda = lowerBound;
		
		// finds the lambda with the minimum perplexity value
		for (double lambda = lowerBound + 0.01; lambda <= upperLimit; lambda += 0.01)
		{
			// create Lidstone object with the current lambda
			Lidstone currentLidstone = new Lidstone(LIDSTONE_TRAIN_FILE_NAME, obsVoc, lambda);
			
			// calculate the object's perplexity
			double currentPerplexity = Parser.perplexity(currentLidstone, LIDSTONE_VALID_FILE_NAME);
			
			if (currentPerplexity < minPerplexity)
			{
				minPerplexity = currentPerplexity;
				bestLambda = lambda;
			}
		}
		
		// return the best lambda value
		return bestLambda;		
	}
	
	/**
	 * Write for every frequency it's expected frequency in the Lidstone model, it's expected frequency
	 * in the Held-Out model, the number of events in it's frequency in the training set of held-out
	 * and the sum of frequencies of those events in the held-out set
	 * @param lidstone model with the best lambda
	 * @param heldOut model
	 * @param OutputMngr outputMngr
	 */
	private static void writeTableResults(Lidstone lidstone, HeldOut heldOut, OutputMngr outputMngr)
	{
		
		// get a map with numOfWords and sumInsances of each frequency in the test set
		Map<Integer, Map.Entry<Integer, Integer>> heldOutResults = heldOut.getInstancesDetails();
		
		for (Map.Entry<Integer, Map.Entry<Integer, Integer>> currEntry : heldOutResults.entrySet())
		{
			// get the current frequency
			int currentFrequency = currEntry.getKey();
			
			// get the the number of events in it's frequency in the training set of held-out
			int numOfevents = currEntry.getValue().getKey();
			
			// get the sum of frequencies of those events in the held-out set
			int sumOfFrequencies = currEntry.getValue().getValue();
			
			// get the expected frequency in the Lidstone model
			double lidstoneExpectedFrequency = lidstone.expectedFrequency(currentFrequency);
			
			// get the expected frequency in the Held-Out model
			double heldOutExpectedFrequency = heldOut.expectedFrequency(currentFrequency);
			
			// round the results to 5 digits after the decimal point
			lidstoneExpectedFrequency = (double)Math.round(lidstoneExpectedFrequency * 100000) / 100000;
			heldOutExpectedFrequency = (double)Math.round(heldOutExpectedFrequency * 100000) / 100000;
			
			// write the current frequency's results
			outputMngr.addLine(currentFrequency + "\t" + lidstoneExpectedFrequency + "\t" + heldOutExpectedFrequency
					+ "\t" + numOfevents + "\t" + sumOfFrequencies);
		}
	}	
}










